package crawl.http;

import index.Document;

import java.io.File;
import java.util.Scanner;
import java.util.Stack;
import java.util.Vector;

import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.nodes.RemarkNode;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.nodes.TextNode;
import org.htmlparser.tags.ScriptTag;
import org.htmlparser.tags.StyleTag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.util.SimpleNodeIterator;

import crawl.DocumentParser;

import text.ChineseStem;
import utils.Logger;
import utils.Util;

/**
 * parser a html page
 * 
 * @author huangcd
 */
public class HtmlParser implements DocumentParser
{
    private static Vector<Class<? extends Node>> ignoreTagList;
    private Parser                               parser;
    private Document                             document;

    static
    {
        ignoreTagList = new Vector<Class<? extends Node>>();
        ignoreTagList.add(StyleTag.class);
        ignoreTagList.add(ScriptTag.class);
    }

    public HtmlParser(int documentID, String charset) throws ParserException
    {
        File d = Util.getDocument(documentID);
        if (d == null)
            throw new ParserException("document does not exists" + documentID);
        this.document = Document.getDocument(documentID);
        parser = new Parser("file:/" + d.toString());
        parser.setEncoding(charset);
    }

    public Document parse()
    {
        NodeIterator iterator;
        try
        {
            iterator = parser.elements();
            if (!iterator.hasMoreNodes())
                throw new ParserException("no element to be parse");
            Node node = iterator.nextNode();
            Stack<String> tagStack = new Stack<String>();
            parseNode(node, tagStack);
            return document;
        }
        catch (ParserException e)
        {
            Logger.logException("" + document.getDocumentID() + " "
                    + e.toString(), true);
            parser.reset();
            return parse();
        }
    }

    private void parseNode(Node node, Stack<String> tagStack)
    {
        Class<? extends Node> nodeClass = node.getClass();
        for (Class<? extends Node> c : ignoreTagList)
        {
            if (nodeClass.equals(c))
            {
                Logger.logInfo("ignore tag " + node.getClass().toString());
                return;
            }
        }
        if (node instanceof TextNode)
        {
            parseTextNode((TextNode) node, tagStack);
        }
        else if (node instanceof RemarkNode)
        {
            parseRemarkNode((RemarkNode) node, tagStack);
        }
        else if (node instanceof TagNode)
        {
            parseTagNode((TagNode) node, tagStack);
        }
    }

    private void parseTextNode(TextNode node, Stack<String> tagStack)
    {
        String text = node.toPlainTextString().replaceAll(
                Util.getHtmlParserIgnorePattern(), "").trim();
        if (text.isEmpty())
            return;
        // 计算特征项一次出现的权值
        double weight = Util.getDefaultTermWeight();
        for (String tag : tagStack)
        {
            weight += TagWeight.getWeight(tag);
        }
        String stemText = ChineseStem.parse(text);
        Scanner scan = new Scanner(stemText);
        while (scan.hasNext())
        {
            String term = scan.next();
            document.addTerm(term, weight);
        }
    }

    private void parseTagNode(TagNode node, Stack<String> tagStack)
    {
        tagStack.add(node.getTagName().toUpperCase());
        NodeList list = node.getChildren();
        if (list != null)
        {
            SimpleNodeIterator iterator = list.elements();
            while (iterator.hasMoreNodes())
            {
                Node child = iterator.nextNode();
                parseNode(child, tagStack);
            }
        }
        tagStack.pop();
    }

    private void parseRemarkNode(RemarkNode node, Stack<String> tagStack)
    {

    }

    // public static void main(String[] args) throws ParserException,
    // MalformedURLException
    // {
    // Document d = (new HtmlParser(1, "UTF-8")).parse();
    // System.out.println(d.getDocumentID());
    // System.out.println(d.getTermCount());
    // System.out.println(d.getContent().size());
    // int moreAppearance = 0;
    // for (Term t : d.getContent())
    // {
    // if (t.getWeight(d) == 1)
    // System.err.println(t.getName());
    // if (t.getCount() > 1)
    // {
    // moreAppearance++;
    // System.out.println(t.getName() + " " + t.getWeight(d));
    // }
    // }
    // System.out.println(moreAppearance);
    // }
}
